import requests
from lxml import etree
import pymysql
import A地区

sql = "select id,href from city"
conn = pymysql.connect(host='localhost', user='root', password='0830', database='moonlightcat_test')
cursor = conn.cursor()

# href = 爬取各个大城市名.bigCity()
# 定义获取大城市表的数据的方法（url和id）
def getUrlAndIdFromCity():
    cursor.execute(sql)
    result = cursor.fetchall()
    return result

def fun():

    headers = {
        "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.54 Safari/537.36 Edg/95.0.1020.40"
     }

    result = A地区.getDatafromCity() #(id,url)
    for data in result:
        city_href = data[1]

        child_resp = requests.get(url=data[1], headers=headers)
        child_resp.encoding = "utf-8"
        small_city_text = child_resp.text
        tree2 = etree.HTML(small_city_text)
        small_city_href = tree2.xpath('//div[@class="cnts"]/ul/li[3]/a/@href')
        small_city_address = tree2.xpath('//div[@class="cnts"]/ul/li[3]/a[@class="tab"]/text()')
        if len(small_city_href) > 0:
            small_city_href.pop(0)
            # small_city_href = result + small_city_href
            small_dict = dict(zip(small_city_address,small_city_href))

            for k,v in small_dict.items():
                region_url = city_href.rstrip("/")+v
                sql = "insert into area(name,cityId,cityHref) values('%s','%s','%s')" % (k, data[0], region_url)
                cursor.execute(sql)
                print("数据插入成功！",region_url)
                conn.commit()

fun()